import datetime
import inspect
import json
import types

from re_common.baselibrary.tools.all_requests.aiohttp_request import AioHttpRequest
from re_common.baselibrary.tools.all_requests.mrequest import MRequest
from re_common.baselibrary.utils.basetime import BaseTime
from re_common.baselibrary.utils.core.mdeprecated import try_except2_async, retry_func_async

from apps.crawler_platform.core_callback.core_callback import core_down_befor_callback, core_down_after_callback
from apps.crawler_platform.core_callback.err_callback import all_client_back_callback
from apps.crawler_platform.core_platform.downler import DownStep
from apps.crawler_platform.core_platform.g_model import PageHtmlModel, HtmlModel, MultiplePageHtmlModel, TypeHtmlModel


class CoreDownMixin(DownStep):
    """
    下载功能
    由于整个下载的逻辑是以返回的第一个布尔值判断是否下载成功的，有时候下载失败
    需要继续下载，比如翻页和错误的状态码，所以抽取这个下载功能，继承类只实现简单的功能

    """

    def func_add(self):
        """
        rrq的方法添加器,通过callbakck的_m_list后缀添加
        :return:
        """
        # 添加验证器
        name = "_".join([self.pm.task_name, self.pm.task_tag, "m", "list"])
        if name in globals():
            lists_m = eval(name)
            for item in lists_m:
                if item in self.pm.funcs:
                    func_m = types.MethodType(eval(item), self.rrq)
                    self.rrq.__dict__.update({item: func_m})

    async def _core_downler(self):
        """
        具体的下载逻辑,核心的下载器
        """
        if inspect.stack()[1].function != "_easy_down":
            raise Exception("不允许_easy_down以外的函数调用_core_downler")
        core_down_befor_callback(self.pm)
        self.rrq = AioHttpRequest()
        # 添加验证器
        self.func_add()
        Lists_middler = []
        self.rrq.set_url(self.pm.url) \
            .set_timeout(self.pm.time_out) \
            .set_header(self.pm.this_header)
        if self.pm.proxy:
            self.rrq.set_proxy(self.pm.proxy)

        if self.pm.resp_middler:
            self.rrq.set_middler_para(self.pm.resp_middler)
            for k, v in self.pm.resp_middler.items():
                Lists_middler.append(eval("self.rrq." + k))

        if self.pm.marks:
            self.rrq.set_marks(self.pm.marks)
            Lists_middler.append(self.rrq.marks_middlerwares)
        self.rrq.set_middler_list(Lists_middler)
        if self.pm.params:
            self.rrq.set_params(self.pm.get_params())
        if self.pm.req_type == "POST":
            if self.pm.post_data:
                data = self.pm.post_data
                if self.pm.post_data_is_json == "1":
                    data = json.dumps(self.pm.post_data, ensure_ascii=False)
                self.rrq.set_data(data)
            bools, dicts = await self.rrq.run(MRequest.POST)
        elif self.pm.req_type == "GET":
            bools, dicts = await self.rrq.run(MRequest.GET)
        else:
            raise Exception("req_type 错误;" + self.pm.req_type)
        core_down_after_callback(self.pm, self.rrq, bools, dicts)
        return bools, dicts

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    @retry_func_async(retry_times=3, sleep_time=1)
    async def _easy_down(self):
        """
        基础下载逻辑，不考虑其他逻辑
        :param keys:
        :return:
        """
        self.step_info = f"获取代理,key is {self.pm.keys};"
        self.rrq = AioHttpRequest()
        bools, err_info = await self.get_proxy(self.pm.task_name, self.pm.task_tag)
        if not bools:
            return bools, err_info
        self.step_info = f"下载网页,key is {self.pm.keys};"
        bools, err_info = await self._core_downler()
        # 下载遇到特殊的状态码允许通过，这些状态码在我们的配置当中
        if not bools and "code" in err_info.keys():
            state_dicts = eval(self.pm.state_key)
            if str(err_info["code"]) in state_dicts["key_state"]:
                return True, err_info
        return bools, err_info

    async def stat_down(self):
        """
        下载失败且有明确的code时会返回true，表示继续下载
        :return:
        """
        bools, err_info = await self._easy_down()
        if not bools:
            self.pm.set_down_err_info(err_info)
        self.pm.down_model.down_dict[self.pm.keys] = HtmlModel(
            html=self.rrq.html,
            state_dict=err_info,
            down_date=str(datetime.datetime.now())
        )
        # 对于不翻页 下载成功就设为True
        if bools:
            self.pm.set_is_his(True)
        # 下载遇到特殊的状态码允许通过
        if "code" in err_info.keys():
            return True, err_info
        return bools, err_info

    async def deal_down(self):
        """
        简单下载，下载完了直接解析并写数据库
        一般的下载有多个url时会循环下载完成后才会调用处理callback和save_sql,
        但是在该模式下每一个url都会调用一次callback
        :return:
        """
        bools, err_info = await self._easy_down()
        if not bools:
            self.pm.set_down_err_info(err_info)
        self.pm.down_model.down_dict[self.pm.keys] = HtmlModel(
            html=self.rrq.html,
            state_dict=err_info,
            down_date=str(datetime.datetime.now())
        )
        if bools:
            self.pm.set_is_his(True)
        else:
            return bools, err_info
        bools, err_info = await self.deal_html()
        if not bools:
            return bools, err_info
        self.step_info = f"中间过程保存sql"
        bools, err_info = await self.save_sql()
        if not bools:
            return bools, err_info
        self.step_info = f"再次获取sql的单条数据"
        bools, err_info = await self.get_sql()
        if not bools:
            return bools, err_info
        return bools, err_info

    async def order_down(self):
        """
        简单的排序下载，错误直接返回错误
        :return:
        """
        bools, err_info = await self._easy_down()
        if not bools:
            self.pm.set_down_err_info(err_info)
        self.pm.down_model.down_dict[self.pm.keys] = HtmlModel(
            html=self.rrq.html,
            state_dict=err_info,
            down_date=str(datetime.datetime.now())
        )
        # 对于不翻页 下载成功就设为True
        if bools:
            self.pm.set_is_his(True)
        return bools, err_info

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def multiple_page_down(self):
        """
        翻页，内部有多个分类需要翻页，而不是只是简单的分页
        :return:
        """
        self.rrq = AioHttpRequest()
        self.rrq.html = None
        if self.pm.turn_page == 5:
            first_page = "1"
        elif self.pm.turn_page == 6:
            first_page = "0"
        else:
            raise Exception("翻页设计不兼容,请更改程序")
        # 初始化，防止down_dict没有keys
        self.pm.down_model.down_dict[self.pm.keys] = MultiplePageHtmlModel[HtmlModel, PageHtmlModel[HtmlModel]](
            first_page=HtmlModel(
                html=self.rrq.html,
                down_date=str(datetime.datetime.now())
            ),
            type_page=TypeHtmlModel[PageHtmlModel[HtmlModel]](
                type_code={}
            )
        )

        # 下载其中一页
        @try_except2_async(callback=all_client_back_callback, is_print=True)
        async def down_first_page(self, page):
            self.pm.set_post_data(str(page))
            self.pm.set_params(str(page))

            bools, err_info = await self._easy_down()
            if not bools:
                self.pm.set_down_err_info(err_info)
            self.pm.down_model.down_dict[self.pm.keys].first_page = HtmlModel(
                html=self.rrq.html,
                state_dict=err_info,
                down_date=str(datetime.datetime.now())
            )
            return bools, err_info

        # 下载其中一页
        @try_except2_async(callback=all_client_back_callback, is_print=True)
        async def down_one_page(self, type_code, page):
            type_page = self.pm.down_model.down_dict[self.pm.keys].type_page
            if str(type_code) in type_page.type_code.keys() and str(page) in type_page.type_code[
                str(type_code)].page_html.keys():
                html = type_page.type_code[str(type_code)].page_html[str(page)].html
                if html and len(html) > 0:
                    # 跳过
                    return True, "continue"

            self.pm.set_post_data(str(page))
            self.pm.set_params(str(page))
            bools, err_info = await self._easy_down()
            if not bools:
                self.pm.set_down_err_info(err_info)
            if bools:
                type_page = self.pm.down_model.down_dict[self.pm.keys].type_page
                if type_code not in type_page.type_code.keys():
                    type_page.type_code[type_code] = PageHtmlModel(page_html={})
                type_page.type_code[type_code].page_html[str(page)] = HtmlModel(
                    html=self.rrq.html,
                    state_dict=err_info,
                    down_date=str(datetime.datetime.now())
                )
            return bools, err_info

        # 保存和查询mongo的必要参数
        self.pm.set_para()
        bools, err_info = await self.read_nosql_html()
        if not bools:
            return bools, err_info
        # 查询结果
        data = self.pm.mongo_html["data"]

        self.pm.sql_model.page = first_page
        # 如果mongo查出来存在
        if data:
            # 注入到model
            data_obj = data[0]["new_data"]
            html = None
            # 防止new_data是空字典
            if data_obj:
                self.pm.down_model = self.pm.down_model.set_parse_obj(data_obj)
                # 第一页的html存在，且为当天的
                first_html_model = self.pm.down_model.down_dict[self.pm.keys].first_page
                if first_html_model.html != "" and BaseTime().get_beijin_date_strins(
                        format="%Y%m%d") < first_html_model.down_date:
                    html = self.pm.down_model.down_dict[self.pm.keys].first_page.html
                    # 启用第一页不信任机制，保证第一页会被重下 20210618
                    html = ""
            # 如果第一页不存在
            if not (html and len(html) > 0):
                # 下载第一页,
                bools, err_info = await down_first_page(self, first_page)
                if not bools:
                    return bools, err_info
        else:
            # 不存在就 下载第一页
            bools, err_info = await down_first_page(self, first_page)
            if not bools:
                return bools, err_info
        # 解析第一页，得到页数
        bools, err_info = await self.deal_html()
        if not bools:
            return bools, err_info

        # 一个标志，判断页是否下载完
        is_his = True
        # 连续下载页数
        down_page_count = 0
        for k, v in self.deal_model.code_dicts[self.pm.keys].items():
            dicts = v["params"]
            page = v["max_page"]
            self.pm.sql_model.set_json(dicts)
            self.pm.sql_model.page = page
            # 启用连续3页下载错误便结束下载机制(20210618)目的是防止第一页网页可能给虚假的页数进行投毒
            page_down_fail = 0
            for page in range(int(first_page) + 1, int(self.pm.sql_model.page) + 1):
                bools, err_info = await down_one_page(self, k, page)
                if err_info != "continue":
                    down_page_count = down_page_count + 1
                if down_page_count >= 10:
                    bools,err_info = await self.save_nosql_html()
                    if not bools:
                        return bools, err_info
                    down_page_count = 0
                if not bools:
                    page_down_fail = page_down_fail + 1
                    is_his = False
                else:
                    page_down_fail = 0
                if page_down_fail > 3:
                    break
        # 设置保存nosql时判定是否下载完
        self.pm.set_is_his(is_his)
        return bools, err_info

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def page_down(self):
        #  TODO 不是好的解决办法,但能解决问题(其他人没有更好办法前不要改动以下两句代码)
        self.rrq = AioHttpRequest()
        self.rrq.html = None
        if self.pm.turn_page == 2:
            first_page = "1"
        elif self.pm.turn_page == 1:
            first_page = "0"
        else:
            raise Exception("翻页设计不兼容,请更改程序")
        # 初始化，防止down_dict没有keys
        self.pm.down_model.down_dict[self.pm.keys] = PageHtmlModel[HtmlModel](
            page_html={first_page: HtmlModel(
                html=self.rrq.html,
                down_date=str(datetime.datetime.now())
            )}
        )

        # 下载其中一页
        @try_except2_async(callback=all_client_back_callback, is_print=True)
        async def down_one_page(self, page):
            page_html_dict = self.pm.down_model.down_dict[self.pm.keys].page_html
            # 如果page存在且不为空
            if str(page) in page_html_dict.keys() and page_html_dict[str(page)].html and len(
                    page_html_dict[str(page)].html) > 0:
                # 跳过
                return True, "continue"
            self.pm.set_post_data(str(page))
            self.pm.set_params(str(page))
            bools, err_info = await self._easy_down()
            if not bools:
                self.pm.set_down_err_info(err_info)
            # 对于翻页，一定是下载成功的才写入mongo,因为要依靠是否有html判断是否下载
            if bools:
                page_html = self.pm.down_model.down_dict[self.pm.keys].page_html
                page_html[str(page)] = HtmlModel(
                    html=self.rrq.html,
                    state_dict=err_info,
                    down_date=str(datetime.datetime.now())
                )
            return bools, err_info

        # 保存和查询mongo的必要参数
        self.pm.set_para()
        bools, err_info = await self.read_nosql_html()
        if not bools:
            return bools, err_info
        # 查询结果
        data = self.pm.mongo_html["data"]

        self.pm.sql_model.page = first_page
        # 如果mongo查出来存在
        if data:
            # 注入到model
            data_obj = data[0]["new_data"]
            html = None
            # 防止new_data是空字典
            if data_obj:
                self.pm.down_model = self.pm.down_model.set_parse_obj(data_obj)
                # 第一页的html
                if first_page in self.pm.down_model.down_dict[self.pm.keys].page_html:
                    html = self.pm.down_model.down_dict[self.pm.keys].page_html[first_page].html
                    # 启用第一页不信任机制，保证第一页会被重下 20210618
                    html = ""
            # 如果第一页不存在
            if not (html and len(html) > 0):
                # 下载第一页,
                bools, err_info = await down_one_page(self, first_page)
                if not bools:
                    return bools, err_info
        else:
            # 不存在就 下载第一页
            bools, err_info = await down_one_page(self, first_page)
            if not bools:
                return bools, err_info
        # 解析第一页，得到页数
        bools, err_info = await self.deal_html()
        if not bools:
            return bools, err_info
        # 设置最大页
        self.pm.sql_model.page = self.deal_model.code_dicts[self.pm.keys]["max_page"]
        # 一个标志，判断页是否下载完
        is_his = True
        # 启用连续3页下载错误便结束下载机制 20210618
        page_down_fail = 0
        # 连续下载页数
        down_page_count = 0
        for page in range(int(first_page), int(self.pm.sql_model.page) + 1):
            bools, err_info = await down_one_page(self, page)
            if err_info != "continue":
                down_page_count = down_page_count + 1
            if down_page_count >= 10:
                bools, err_info = await self.save_nosql_html()
                if not bools:
                    return bools, err_info
                down_page_count = 0
            if not bools:
                page_down_fail = page_down_fail + 1
                is_his = False
            else:
                page_down_fail = 0
            if page_down_fail > 3:
                break
        # 设置保存nosql时判定是否下载完
        self.pm.set_is_his(is_his)
        return bools, err_info

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def part_page_down(self):
        self.rrq = AioHttpRequest()
        self.rrq.html = None
        # 保存和查询mongo的必要参数
        bools, err_info = await self.order_down()
        return bools, err_info





